import requests
from datetime import datetime
from lxml import etree
from tinydb import where
from common import UID,HandleTmpList,parseContentToName,ProductToGroup,sess
SPIDERNAME='隆众资讯'

def parseLongZhongPubTime(time_str):
    #[2020-11-17 19:41]
    ls=time_str.split('\n')
    s=ls[0].strip('[')+' '+ ls[1].strip(']').strip()+':00'
    return s

def parseLongZhongContent(url):
    r=sess.get(url)
    r.encoding='utf8' 
    selector=etree.HTML(r.text)
    ele=selector.cssselect("#content")[0]
    content=ele.xpath("string(.)").strip()
    # print(content)
    return content
def getLongZhongArticleList(articleCol,BeCrawledUrlList):
    productList=[
        'https://list.oilchem.net/384/3287/',#纯碱
        'https://list.oilchem.net/188/2700/',#尿素
        'https://list.oilchem.net/454/255/',#液化气
        'https://list.oilchem.net/140/18263/',#甲醇
        'https://list.oilchem.net/2959/28094/',#纸浆
        'https://list.oilchem.net/143/2219/',#PTA
        'https://list.oilchem.net/113/3296/',#玻璃
    ]
    temp_article_ls=[]
    for url in productList:

        r=sess.get(url)
        selector=etree.HTML(r.text)
        print(url)
        for ele in selector.cssselect('.list ul ul li'):
            articleUrl=ele.xpath('./a/@href')[0]
            #判断是否已经爬取过，如果是，跳出循环
            if len(articleCol.search(where('url')==articleUrl)):break
            title=ele.xpath('./a')[0].text
            publicTime=parseLongZhongPubTime(ele.xpath('./span/text()')[0])
            temp_dict={'tags':['隆众资讯'],'score':0,'uid':UID()}
            temp_dict['title']=title.strip()
            temp_dict['articleFrom']=SPIDERNAME
            temp_dict['url']=articleUrl.strip()
            temp_dict['publicTime']=publicTime.strip()

             #文章内容
            content=parseLongZhongContent(articleUrl)
            temp_dict['content']=content
            #定文章所属期货品种,板块
            n=parseContentToName(title+content)

            if n:
                print(SPIDERNAME,'   ',title,"  ",n)
                temp_dict['product_name']=n
                temp_dict['group']=ProductToGroup[n]
            else:
                print("………………………………未找到品种名称，可能异常")
                temp_dict['product_name']=''
                temp_dict['group']=''
            
            temp_article_ls.append(temp_dict)
    
    #注意缩进不要错
    HandleTmpList(temp_article_ls,articleCol,SPIDERNAME)